open Glob.Bin

module type Get_event = 
    sig
        type serialized
        val n_events: int64
(*        val fill: int64 -> serialized -> unit*)
        val get: int64 -> serialized 
    end

let format n = "ev"^(Int64.to_string n)

open Printf
let command text =
    let out = Sys.command text in
    if out <> 0 then (
        failwith (sprintf 
        "command \"%s\" exited with return code %i" text out))
    else ()
;;

let file_formats = [ "Normal_file"; "Gzipped_file"; ]

let gzip file_name = command ("gzip -f "^file_name)
let gunzip file_basename = 
    command ("gunzip -c "^file_basename^".gz > "^file_basename)

module WRITE_EVENTS (Data: Get_event) = 
struct
    module Db = Ocaml_dbm.Make (Data)
(*    let convert serialized dir_name =*)
    let write ?(file_format="Gzipped_file") dir_name =
        if Sys.file_exists dir_name then (
            command ("rm -rf "^dir_name));
        Unix.mkdir dir_name 0o744;
        Unix.chdir dir_name;
        let info_file = open_out "info.txt" in
        fprintf info_file "n=%Li format=%s\n" Data.n_events file_format;
        let n = ref 0L in
        let current_file_No = ref 0 in
        while !n < Data.n_events do
            let file_name = sprintf "f_%i" !current_file_No in
            let db = Db.opendbm file_name 
                [Dbm.Dbm_rdwr; Dbm.Dbm_create] 0o644
            in
            fprintf info_file "%s %Li " file_name !n;
            let size () = 
                let st = Unix.stat (file_name^".pag") in
                st.Unix.st_size
            in
            (* gdbm file size should be < 350M for effecient work *)
            while !n < Data.n_events && size () < 350_000_000 do
                let serialized = Data.get !n in
                Db.replace db (format !n) serialized;
                if Int64.rem !n 1_000L = 0L then (
                    Printf.printf "filled event No~%Ld \n" !n;
                    flush stdout);
                n := Int64.succ !n
            done;
            Db.close db;
            fprintf info_file "%Li\n" (Int64.add !n (-1L));
            if file_format = "Gzipped_file" then (
                command ("rm -f "^file_name^".dir");
                gzip (file_name^".pag")
            );
            incr current_file_No
        done;
        close_out info_file
end
;;

module READ_EVENTS (Data: Ocaml_dbm.Serialized_type) =
struct
    open Printf
    open Scanf
    module Db = Ocaml_dbm.Make (Data)
    type file = {mutable db: Db.t option; n1: int64; n2: int64; name: string}
    type opt_proc = { search_gunzipped : bool }
    type descr = { top_dir: string; file_format: string; 
            n_events: int64; files: file list; opt: opt_proc }
    let get_db file = file.db
    let clean file =
        if Sys.file_exists file 
        then (Sys.remove file) else ()
    let cleanup data_info =
        List.iter (fun file ->
            if data_info.file_format = "Gzipped_file" then (
                List.iter (fun suffix -> 
                        clean (data_info.top_dir // file.name ^ suffix)) 
                    [ ".dir"; ".pag" ]
            )
        ) data_info.files

    exception File_not_found
    let open_file ~file ~file_format ~dir_name:top_dir ~opt =
        let file_basename = top_dir // file.name in
        let file_pag = file_basename^".pag" in
        if file_format = "Gzipped_file" then (
            if not opt.search_gunzipped then (
                print_endline "removing old file...";
                clean file_basename);
            if not (Sys.file_exists file_pag) then (
                if opt.search_gunzipped then (
                    print_endline "file not found. Trying gzipped...");
                gunzip file_pag;
            );
            if Sys.file_exists file_pag then (
                print_endline "OK"
            ) else (
                print_endline ("File "^file_pag^" not found!");
                raise File_not_found);
        );
        Db.opendbm file_basename [Dbm.Dbm_rdonly] 0o644
        
    let get_info ?(search_gunzipped = false) dir_name =
        let info_file = open_in (dir_name // "info.txt") in
        let n_events, file_format = 
            fscanf info_file "n=%Li format=%s\n" (fun a b -> a, b) in
        if not (List.mem file_format file_formats) then (
            failwith "unknown file format in info.txt");
        let opt = {search_gunzipped = search_gunzipped} in
        let rec next_file list =
(*            printf "scan nf=%i\n" (List.length list); flush stdout;*)
            let (file_name,n1,n2) =
                fscanf info_file "%s %Li %Li" (fun name n1 n2 -> (name,n1,n2))
            in 
            let current_file = {db=None;n1=n1;n2=n2;name=file_name} in
(*            let db = if search_gunzipped 
            then Some (open_file ~file:current_file ~file_format ~dir_name ~opt)
            else None in
            current_file.db <- db;*)
            let current_list = current_file::list in
            if Int64.sub n_events 1L = n2
            then current_list
            else ( fscanf info_file "\n" (); next_file current_list)
        in
        let file_info' = try (next_file [])
        with End_of_file -> (failwith "unexpected end of ocaml dbm-info!")
        in 
        let info = { top_dir = dir_name; file_format = file_format; 
           n_events = n_events; files = List.rev file_info';
           opt = opt } in
(*        cleanup info;*)
        info

    let get n data_info = 
        if n < 0L || n >= data_info.n_events
        then (failwith "wrong event number demanded")
        else (
(*            printf "get n=%Li\n" n;
            print_endline "hi1!";
            List.iter (fun file -> printf "n1=%Li n2=%Li n=%Li\n" file.n1
            file.n2 n;flush stdout) data.files;*)
(*            print_endline "hi2!";*)
            let file = List.find
                (fun file -> 
(*                    printf "n1=%Li n2=%Li n=%Li\n" file.n1 file.n2 n;
                    flush stdout;*)
                    file.n1<=n && n<=file.n2) data_info.files in
            if (Int64.rem n 1000L = 0L) || (n = Int64.sub n 1L) then (
                print_endline ("find "^(format n)^" in file "^file.name));
            let db = ( match file.db with
                | Some db -> db
                | None -> (* file was not opened previously *)
                        let db = open_file ~file ~dir_name:data_info.top_dir
                                ~opt:data_info.opt 
                                ~file_format:data_info.file_format in
                        file.db <- Some db;
                        db
            ) in
            Db.find db (format n) 
        )
    ;;
    let iterl f data_info =
        Glob.iter64 ~range:(0L,data_info.n_events)
                ~f:(fun n -> f n (get n data_info))

    let iter f data_info =
        Glob.iter64 ~range:(0L,data_info.n_events)
                ~f:(fun n -> f (get n data_info))

  let close ?(not_remove = false) data_info =
      if not not_remove then (
          cleanup data_info);
      List.iter (fun file ->
          match file.db with
             | None -> ()
             | Some db -> 
                 Db.close db; file.db <- None
      ) data_info.files
end

